{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d6c5c615",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import yaml\n",
    "from types import SimpleNamespace\n",
    "from collections import OrderedDict\n",
    "\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from PIL import Image\n",
    "from torch.utils.data import DataLoader, WeightedRandomSampler, Dataset\n",
    "from torch import optim\n",
    "import torchvision.transforms as T\n",
    "from sklearn import metrics\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from data.datasets import AortaDataset3DCenter, get_weight_list, AortaDataset\n",
    "from utils.ranger import Ranger\n",
    "from utils.lr_scheduler import CosineAnnealingWithWarmUpLR\n",
    "from model.resnet3d import resnet3d\n",
    "from model.SupCon import resnet\n",
    "import data.transforms as MT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e6b5a19a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#os.environ['CUDA_VISIBLE_DEVICES'] = '1'\n",
    "device = torch.device('cuda:2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "723b47a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#2d数据\n",
    "image_size = 81\n",
    "source = '/nfs3-p2/zsxm/dataset/aorta_classify_ct_-100_500/'\n",
    "cate = ['0','1']\n",
    "n_classes = len(cate)\n",
    "sobel = True\n",
    "vt_list = [\n",
    "    T.Resize(image_size),\n",
    "    T.CenterCrop(image_size),\n",
    "    T.ToTensor(),\n",
    "]\n",
    "if sobel:\n",
    "    vt_list.append(MT.SobelChannel(3))\n",
    "val_transform = T.Compose(vt_list)\n",
    "test_dataset = AortaDataset(os.path.join(source, 'test'), cate, val_transform)\n",
    "test_loader = DataLoader(test_dataset,\n",
    "                         batch_size=128,\n",
    "                         shuffle=False,\n",
    "                         drop_last=False,\n",
    "                         num_workers=8, \n",
    "                         pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "87fc955b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "#2d网络\n",
    "load_model = '.details151/CE/12-05_18_36_21_有Sobel,只分类阴性和夹层/Net_best.pth'\n",
    "net = resnet(34, n_channels=2 if sobel else 1, n_classes=n_classes)\n",
    "net.load_state_dict(torch.load(load_model, map_location=device))\n",
    "net.to(device)\n",
    "net.eval()\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10f32662",
   "metadata": {},
   "outputs": [],
   "source": [
    "#3d数据\n",
    "image_size = 81\n",
    "source = '/nfs3-p2/zsxm/dataset/aorta_classify_ct_-100_500/center'\n",
    "cate = ['0','1']\n",
    "n_classes = len(cate)\n",
    "sobel = True\n",
    "vt_list = [\n",
    "    MT.Resize3D(image_size),\n",
    "    MT.CenterCrop3D(image_size),\n",
    "    MT.ToTensor3D(),\n",
    "]\n",
    "if sobel:\n",
    "    vt_list.append(MT.SobelChannel(3, flag_3d=True))\n",
    "val_transform = T.Compose(vt_list)\n",
    "test_dataset = AortaDataset3DCenter(os.path.join(source, 'test'), cate, val_transform, depth=7, step=1)\n",
    "test_loader = DataLoader(test_dataset,\n",
    "                         batch_size=128,\n",
    "                         shuffle=False,\n",
    "                         drop_last=False,\n",
    "                         num_workers=8, \n",
    "                         pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e456e4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#3d网络\n",
    "load_model = '.details151/CE3D/09-12_180348_3D,有Sobel,只分类阴性和夹层/Net_best.pth'\n",
    "net = resnet3d(34, n_channels=2 if sobel else 1, n_classes=n_classes, conv1_t_size=3)\n",
    "net.load_state_dict(torch.load(load_model, map_location=device))\n",
    "net.to(device)\n",
    "net.eval()\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f6a82dc6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                                                                                                                                                "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.9460    0.8844    0.9142      3504\n",
      "           1     0.8457    0.9261    0.8841      2396\n",
      "\n",
      "    accuracy                         0.9014      5900\n",
      "   macro avg     0.8958    0.9053    0.8991      5900\n",
      "weighted avg     0.9052    0.9014    0.9019      5900\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "tot_loss = 0\n",
    "true_list = []\n",
    "pred_list = []\n",
    "pred_ori_list = []\n",
    "with tqdm(total=len(test_dataset), desc=f'test round', unit='img', leave=False) as pbar:\n",
    "    for imgs, labels in test_loader:\n",
    "        imgs, labels = imgs.to(device), labels.to(device)\n",
    "        preds = net(imgs)\n",
    "        if n_classes > 1:\n",
    "            tot_loss += F.cross_entropy(preds, labels).item() * labels.size(0)\n",
    "            pred_idx = torch.softmax(preds, dim=1)\n",
    "        else:\n",
    "            tot_loss += F.binary_cross_entropy_with_logits(preds.squeeze(1), labels.float()).item() * labels.size(0)\n",
    "            pred_sig = torch.sigmoid(preds)\n",
    "            pred_idx = torch.cat([1-pred_sig, pred_sig], dim=1)\n",
    "        pred_ori_list += pred_idx.tolist()\n",
    "        pred_idx = pred_idx.argmax(dim=1)\n",
    "        labels_list = labels.tolist()\n",
    "        true_list += labels_list\n",
    "        pred_idx = pred_idx.tolist()\n",
    "        pred_list.extend(pred_idx)\n",
    "        pbar.update(labels.size(0))\n",
    "tot_loss /= len(test_dataset)\n",
    "\n",
    "true_list, pred_list, pred_ori_list = np.array(true_list),np.array(pred_list),np.array(pred_ori_list)\n",
    "\n",
    "print(f'Test report:\\n'+metrics.classification_report(true_list, pred_list, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "275df6c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9013559322033898 0.926126878130217 0.884417808219178\n"
     ]
    }
   ],
   "source": [
    "print(metrics.accuracy_score(true_list, pred_list),metrics.recall_score(true_list, pred_list, pos_label=1),metrics.recall_score(true_list, pred_list, pos_label=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "91339df2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9653\n"
     ]
    }
   ],
   "source": [
    "print(f'{metrics.roc_auc_score(true_list, pred_ori_list[:,1]):.4f}')\n",
    "#print(metrics.roc_auc_score(1-true_list, pred_ori_list[:,0]))\n",
    "\n",
    "x,y,thresh = metrics.roc_curve(true_list, pred_ori_list[:,1])\n",
    "# roc_root = './roc_curve/ct2d'\n",
    "# os.makedirs(roc_root, exist_ok=True)\n",
    "# torch.save([x,y], os.path.join(roc_root, '5.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b051f13d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "892"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(thresh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1100dd53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGoCAYAAADvp1oKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABLtUlEQVR4nO3deXxU5dn/8c+VAIbNDaUqO6Iiu4KAioAoIigoirIJgepjfR61tv3Z7alVa221tbXax63uCbsoKsoibogLyqLsKIsgBkQBAdnJcv3+mJk4hEkyCZlMZub7fr3mlZkzZ7nmJJwv9zln7tvcHRERkXhIi3cBIiKSuhRCIiISNwohERGJG4WQiIjEjUJIRETiRiEkIiJxoxASqWLM7L/N7Fsz221m9eJdj0gsKYQkaZnZWDP7xsx+MLNVZnZD2Hs9zawgeKDfbWY5ZvaCmZ1Twvqampmb2bQI27k7yprWm9nFJbxfHXgQuMTd67j7tmjWG8V2h5nZguBn/cbMZphZNzN7ImwfHDSz3LDXM4LLmpl9aWYrKqIWkXAKIUlm9wFN3f1oYABwr5l1DHt/k7vXAeoCXYHPgffN7KJS1tvVzM6PScXwEyADWF7WBYNhcdi/aTP7FfAQ8Nfg+hsDjwFXuPtNwbCrE3x/Uui1u/cNrqI7UB9oXlJIi5SHQkiSlrsvd/cDoZfBx6kR5nN3z3H3O4Gngb+Vsuq/A/cW96aZXW5mi8xsh5l9ZGbtgtPHEAiA14Itjd8UWe504Ivgyx1m9k5w+nlmNt/MdgZ/nhe2zGwz+4uZfQjsBZoXWecxwD3Aze4+xd33uHuuu7/m7r8u5XOGZAKvAtODz0UqjEJIkpqZPWZmewm0cr4hcCAtyRTgbDOrXcI8jwKnRzqtZmZnA88CPwPqAf8BpprZUe4+AtgA9A+2NP4evqy7rwJaB18e6+69zOx4YBrw7+D6HgSmFblWNAK4kUCL7qsiJZ1LoGX1cimfOyIzqwUMAsYFH0PMrEZ51iUSiUJIkpq7/w+Bg/MFBALmQMlLsAkw4NgS5tkP/IXIraH/Av7j7p+4e767ZwW32bWMpYdcBqx29zHunufuEwgEav+weZ4Ptvry3D23yPL1gK3unlfO7V9FoP5ZwOtAtWBNIhVCISRJLxgGHwANgf8uZfYGBE7b7ShlvqeAn5hZ/yLTmwD/L3gqboeZ7QAaAaeUufCAUzi8dfNVsM6Qr0tYfhtwgplVK+f2M4EXggF3gECQ65ScVBiFkKSSakS4JlTEQOBTd99T0kzBFsefgD8TaDmFfA38xd2PDXvUCrZgIBBwZbGJQLCFawxsDC+nhOXnEmi5XVnG7WJmDYFewHVmttnMNhM4NdfPzE4o6/pEIlEISVIys/pmNsTM6phZupn1AYYC70SY18ysgZndBdwA/G+UmxkDHAVcGjbtKeAmM+sSXG9tM7vMzOoG3/+WIjcPlGI6getPw8ysmpkNBloRODVWKnffCdwJPGpmV5pZLTOrbmZ9zezvpSw+AlgFnAF0CD5OB3II7EuRI6YQkmTlBE695QDbgX8Av3D3V8PmOcXMdgO7gflAW6Cnu8+KagPu+cBdwPFh0xYQuC70SHC7a4BRYYvdB9wRPFV3exTb2AZcDvw/AqfWfgNc7u5bo6kxuI4HgV8BdwBbCLTWbgFeKWXRTOAxd98c/gCeQKfkpIKYBrUTEZF4UUtIRETiRiEkIiJxoxASEZG4UQiJiEjclPcLbHFzwgkneNOmTeNdhoiIRGHhwoVb3f3E4t5PuBBq2rQpCxYsiHcZIiISBTMr2uPHIXQ6TkRE4kYhJCIicROzEDKzZ83sOzNbVsz7Zmb/NrM1ZrYk2AW+iIikkFi2hJ7n0D61iuoLnBZ83Ag8HsNaRESkCorZjQnuPsfMmpYwyxVAtgf6DfrYzI41s5Pd/ZtY1VSZ9owdx95XXjlk2nst8vikWR5d9n7FWfs2Rl6wIrlFmAaHdvr8o8KpbpGXjbhMCfMd0iOUYW5l7kK6MgRqiu7zlm2dVU3Ffkap6sr5+y7HH+8WP4UzfjGb446vWeZl43l3XAMOHQclJzjtsBAysxsJtJZo3LhxuTY2c/0M9n38MO03r4j4vuUbOFhB2AHYwfIMz8/Hch0wMC/m4G5YQTpuwd+ge6CNZz/O29GcjhuMZtXXAfDVweaFv3AD0vKrUZBWUPi66EE8LT8NDLzwsBk4qP8YHmFzRxkiIiJHqva+49i0cUfChVCko2TEDHb3J4EnATp16lRqTs9cP4M5X79Hhw2fctY3n5O+L51TC3I5zQOtjw37W1BgTnpBGul56eUq3iMc+c0NN6cgLXKJaQbuxoaDZ7L8h/NYvOOiwqVDYVIj39mZkR5ssFhhwyWwPaNWbgHba1UL5CFQ4JDnTq2jqmHB+c3Azah+MJ/9tarhZoF1BWsuzMXQk+C6sR+f791zkGPr1w4u++Msh7R90gonhq0zOIcFP1PwDSOQ3/k1D/2TCy12MK+A6tXSODqj+o9lHfLTSLND12/BagzYl5tPnYzq1Kye9uM2LbB9C1tH4XL247YNOJDn1KtTI/B++HZCtYd9FsJ+hNZP+PrMyCso4Nia0Y2CbWU6KR5lC7W42SK8Ee1/V4pdZ+S5o1p5WdZZ2qxW9A8nynVGNXekmYpZsPT1Ffk3E83mo525ovZn2JslbXvl8hWs+Hwlffr0oXWL8g0xFc8QyiEw4mRIQwIDeB2RmetnsHXSfdy0/RsaZ6wBYNOulkAGm2jJmu3nsnJbL/ZVMw5Ud47KLyA97SBUy6WgWl6gbWEFkFaAWUHh6/0tTqN5j46k10gnLXhwK/qzwJ36R2dQLc1ISzOOrVXjxwPVIQc/o6PBOUUOYKED2tE1q5OWppaMiFRdc+fOZcPGJdx8SybHHHNMudcTzxCaCtxiZhOBLsDOI70eNGPddKr/fRzX/eR9yICc3S3J2dyM73PP5tSCjdTwPFr37UX366+rkA8gIpKKPvroIxYsWEBm5pEFEMQwhMxsAtCTwPj2OQQG/6oO4O5PEBgxsh+BQb/2AqOPeKOTnuHin0wB4M2dA2n/dTptr7yS2tcNP+JVi4hIxQYQxPbuuBKH/w3eFXdzRW6zzQ+LoQaM2ZvJiH89X5GrFhFJeR9++CELFy6ssACCBOw7rjg+/z80qrGWL3Obk9a4f7zLERFJKh988AGfffYZo0aN4uijj66w9SZNtz25bz4EwNz0Vgz5n6viW4yISBIJBVBmZmaFBhAkUwh9/wObdrXk7TqtSdedZSIiFeL999+PWQBBEoXQ7hrBb9zUqprfVRcRSTTvv/8+ixcvrvBTcOGSJoT2Bb8XeMxRLeJbiIhIEpgzZw6LFy8mMzOTunXrxmw7SXNjQqg7gFMy1Bm3iMiReO+991i6dGnMAwiSqCVkBYHud/J0OUhEpNzee+89li1bVikBBEnUEgp1HNr85NictxQRSXazZ89m+fLlZGZmUqdOnUrZZtK0hAo7Ai1bL4siIkJ8AgiSqCUU8pNjM+JdgohIwnB3Zs+ezcqVKys9gCCpQijQAkpTS0hEJCpFA6h27dqVXkMShVBA3Yzq8S5BRKTKc3feffddvvjii7gFECRhCFVPV0tIRKQk7s4777zDqlWrGDlyZNwCCJLwxgSdjhMRKV5VCiBIwpaQRiQVEYnM3Xn77bdZs2YNmZmZ1KpVK94lKYRERFKBu/PWW2+xdu1aRo4cWSUCCJLodFyILgmJiByqqgYQJGFLSMM4iIj8yN158803WbduXZULIEjCENLpOBGRAHdn1qxZrF+/npEjR1KzZs14l3SYpDsdVy0t6T6SiEiZhQLoq6++qrIBBEnYEqpeTS0hEUlt7s4bb7zBhg0bGDFiRJUNIEjClpCISCoLBdDXX39d5QMIkrAlJCKSqtydmTNnkpOTw4gRI8jIqPodOqslJCKSBBIxgEAtIRGRhOfuzJgxg02bNiVUAIFCSEQkobk706dP55tvvuG6665LqAACnY4TEUlYoQDavHlzQgYQqCUkIpKQ3J1p06bx7bffct1113HUUUfFu6RyUQiJiCQYd+f1119ny5YtCR1AoNNxIiIJJTyAhg8fntABBGoJiYgkDHfntddeY+vWrUkRQKCWkIhIQggF0LZt25ImgEAtIRGRKs/dmTp1Kt9//z3Dhw+nRo0a8S6pwiRdS8hQB6YikjxCAbR9+/akCyBQS0hEpMoqKCjgtddeY/v27QwbNizpAggUQiIiVVJBQQFTp05l586dSRtAoBASEalyCgoKePXVV/nhhx8YOnRo0gYQKIRERKqU8AAaNmwY1atXj3dJMaUQEhGpIgoKCnjllVfYvXt3SgQQJOHdcSIiiSgUQHv27GHo0KEpEUCglpCISNwVFBTw8ssvs3fvXoYMGZIyAQRqCYmIxFUqBxCoJSQiEjcFBQVMmTKF/fv3p2QAgUJIRCQuigZQtWqpeThOuk9tpm57RKRqy8/PZ8qUKRw8eDClAwiS8ZqQMkhEqrDwABo8eHBKBxAkYUtIRKSqys/P56WXXiIvL08BFJR8LSERkSooPICuvfZaBVCQ9oKISIzl5+fz4osvUlBQoAAqQi0hEZEYCg+ga665RgFUhPaGiEiM5OfnM3nyZACuvfZa0tPT41xR1aOWkIhIDIQH0DXXXKMAKoZaQiIiFSwvL4/JkyeTlpbGoEGDFEAlUEtIRKQC5eXl8cILLyiAoqSWkIhIBQkFULVq1bj66qsVQFFQS0hEpAIogMon6UJIvfaISGXLy8tj0qRJVK9eXQFURkkXQiIilSkUQDVq1OCqq65SAJVR8l0TUlNIRCpJXl4eEydOJCMjg6uuuoq0NP2/vqySL4RERCpBbm4ukyZNombNmgwcOFABVE7aayIiZZSbm8vEiRMVQBUgpnvOzC41sy/MbI2Z/S7C+8eY2WtmttjMlpvZ6FjWIyJypEIBVLt2bQVQBYjZ3jOzdOBRoC/QChhqZq2KzHYzsMLd2wM9gX+aWY1Y1SQiciRyc3OZMGECtWvX5sorr1QAVYBY7sHOwBp3/9LdDwITgSuKzONAXQuMyV0H+B7Ii2FNIiLlEgqgunXrKoAqUCz3YgPg67DXOcFp4R4BzgQ2AUuB29y9oOiKzOxGM1tgZgu2bNkSq3pFRCI6ePAg48ePp27dulxxxRUKoAoUyz0Z6WZpL/K6D7AIOAXoADxiZkcftpD7k+7eyd07nXjiiRVdp4hIsQ4ePMiECRM45phjFEAxEMu9mQM0CnvdkECLJ9xoYIoHrAHWAS1jWJOISNRCLaBjjjmGAQMGKIBiIJZ7dD5wmpk1C95sMASYWmSeDcBFAGb2E+AM4MsY1iQiEpVQAB133HEKoBiK2ZdV3T3PzG4B3gDSgWfdfbmZ3RR8/wngz8DzZraUwOm737r71iPZrqnHBBE5QgcPHmTcuHEcf/zxDBgwANOBJWZi2mOCu08HpheZ9kTY803AJbGsQUSkLBRAlSvpuu0xdR4nIuV04MABxo8fT7169ejfv78CqBLoJKeICIEAGjduHCeccIICqBIphEQk5YUC6MQTT+Tyyy9XAFUihZCIpLQDBw4wduxY6tevrwCKg6S7JiQiEq39+/czbtw4TjrpJPr166cAigO1hEQkJe3fv5+xY8cqgOJMISQiKScUQCeffLICKM50Ok5EUkoogE455RT69u2rAIoztYREJGXs37+fMWPG0KBBAwVQFaGWkIikhH379jF27FgaNmzIpZdeqgCqIpKwJaQ/LBE51L59+xgzZgyNGjVSAFUxagmJSFILBVCTJk245JJLFEBVTBK1hPSHJSKH2rdvH9nZ2QqgKizpWkL6GxMR+DGAmjVrRu/evRVAVVTShZCIyN69exkzZowCKAEk0ek4EZFAAGVnZ9O8eXMFUAJQS0hEkkYogFq0aMFFF12kAEoAagmJSFJQACUmtYREJOHt2bOH7OxsTj/9dHr16qUASiAKIRFJaKEAOuOMM7jwwgsVQAlGp+NEJGHt2bOHrKwsBVACU0tIRBLS7t27yc7O5swzz6Rnz54KoASVfCGkv0ORpBcKoFatWtGzZ894lyNHQKfjRCSh7N69m6ysLAVQklAIiUjCCAVQ69atFUBJQqfjRCQh7Nq1i+zsbNq0aUOPHj3iXY5UkBJDyMwaAkOAC4BTgH3AMmAaMMPdC2JeoYikvF27dpGVlUW7du3o3r17vMuRClRsCJnZc0AD4HXgb8B3QAZwOnAp8Acz+527z6mMQkUkNSmAkltJLaF/uvuyCNOXAVPMrAbQODZliYjADz/8QFZWFh06dOCCCy6IdzkSA8XemBAeQGZW08zOKPL+QXdfE8viRCR1hQLorLPOUgAlsVLvjjOzAcAiYGbwdQczmxrjukQkhYUHULdu3eJdjsRQNLdo3wV0BnYAuPsioGnMKhKRlBYKoLPPPlsBlAKiCaE8d98Z80pEJOXt3LmT559/nrPPPpvzzz8/3uVIJYjme0LLzGwYkG5mpwE/Bz6KbVkikmp27txJVlYWnTp14rzzzot3OVJJomkJ3Qq0Bg4A44GdwG2xLOpI6LuqIoknFEDnnHOOAijFRNMSuszd/wD8ITTBzK4BJsesKhFJGTt27CArK4vOnTtz7rnnxrscqWTRtIR+H+U0EZEyCQVQly5dFEApqqQeE/oC/YAGZvbvsLeOBvJiXZiIJLfwAOratWu8y5E4Kel03CZgATAAWBg2fRfwy1gWdWR0VUikqgsFUNeuXenSpUu8y5E4KjaE3H0xsNjMxrt7biXWJCJJbPv27WRlZXHeeefRuXPneJcjcRbNjQlNzew+oBWBDkwBcPfmMatKRJKSAkiKiubGhOeAxwlcB7oQyAbGxLIoEUk+33//PVlZWZx//vkKICkUTQjVdPe3AXP3r9z9bqBXbMsSkWTy/fffk52dzfnnn88555wT73KkConmdNx+M0sDVpvZLcBGoH5syxKRZBFqAV1wwQV06tQp3uVIFRNNS+gXQC0C3fV0BEYAI2NYk4gkiW3btpGVlUX37t0VQBJRqS0hd58ffLobGG1m1YDBwCexLExEEtu2bdvIzs6me/fudOzYMd7lSBVVbEvIzI42s9+b2SNmdokF3AKsAa6tvBLLxvQ1IZG4CwVQjx49FEBSopJaQmOA7cBc4Abg10AN4MrgmEIiIofZunUr2dnZ9OzZk7PPPjve5UgVV1IINXf3tgBm9jSwFWjs7rsqpTIRSTihALrwwgs566yz4l2OJICSbkwo7CXB3fOBdQogESmOAkjKo6SWUHsz+yH43ICawdcGuLsfHfPqykHXhEQq35YtWxgzZgy9evWiQ4cO8S5HEkhJfcelV2YhIpKYtmzZQnZ2NhdddJECSMosmi+riohEFAqgiy++mPbt28e7HElACiERKZfvvvuOMWPG0Lt3b9q1axfvciRBRdNjgojIIRRAUlGiCiEza2JmFwef1zSzurEtS0SqqlAAXXLJJQogOWKlhpCZ/RfwIvCf4KSGwCsxrElEqqhvv/22MIDatm0b73IkCUTTEroZOB/4AcDdV6NetEVSzrfffsvYsWPp06ePAkgqTDQhdMDdD4ZeBDsw9diVdGQMfVFIpKJt3ryZMWPG0KdPH9q0aRPvciSJRBNC75nZ/xL4smpvYDLwWmzLEpGqYvPmzYwdO5a+ffsqgKTCRRNCvwO2AEuBnwHTgTtiWZSIVA3hAdS6det4lyNJKJrvCV0BZLv7U2VduZldCjwMpANPu/v9EebpCTwEVAe2unuPsm5HRCreN998w7hx4+jXrx+tWrWKdzmSpKJpCQ0AVpnZGDO7LHhNqFRmlg48CvQFWgFDzaxVkXmOBR4DBrh7a+CashQvIrGhAJLKUmoIuftooAWBa0HDgLXBoR1K0xlY4+5fBm9smEigVRVuGDDF3TcEt/VdWYoXkYoXCqDLLrtMASQxF9WXVd09F5hBIEgWcniYRNIA+DrsdU5wWrjTgePMbLaZLTSzkZFWZGY3mtkCM1uwZcuWaEoWkXLYtGlTYQCdeeaZ8S5HUkA0X1a91MyeJzCs9yDgaeDkKNYd6V7pord2VwM6ApcBfYA/mtnphy3k/qS7d3L3TieeeGIUmxaRsgoF0OWXX64AkkoTzfWdUQRaQD9z9wNlWHcO0CjsdUNgU4R5trr7HmCPmc0B2gOryrAdETlCGzduZPz48fTv35+WLVvGuxxJIdFcExri7q+UMYAA5gOnmVkzM6sBDAGmFpnnVeACM6tmZrWALsDKMm5HRI5AKIAGDBigAJJKV2xLyMw+cPduZraLQ0+jRTWyqrvnmdktwBsEbtF+1t2Xm9lNwfefcPeVZjYTWAIUELiNe9kRfiYRiVJOTg4TJkxgwIABnHHGGfEuR1JQSSOrdgv+LHeP2e4+ncCXW8OnPVHk9QPAA+XdhoiUTyiArrjiCk4//bBLsSKVIpobE8ZEM63KUNdxIqVSAElVEc2NCYf01RH8smrH2JQjIrH29ddfM3HiRK688kpOO+20eJcjKa7YlpCZ/T54Paidmf0QfOwCviVwQ4GIJBgFkFQ1xYaQu98XvB70gLsfHXzUdfd67v77SqxRRCrAhg0bmDhxIgMHDlQASZVR0t1xLd39c2CymZ1d9H13/zSmlYlIhdmwYQOTJk1i4MCBtGjRIt7liBQq6ZrQr4AbgX9GeM+BXjGp6AiZ6c4EkXBfffUVL7zwAldddRWnnnpqvMsROURJt2jfGPx5YeWVIyIVSQEkVV00t2hfY2Z1g8/vMLMpZnZW7EsTkSMRCqCrr75aASRVVjS9aP/R3XeZWTcCnYxmAU+UsoyIxNH69esLA6h58+bxLkekWNGEUH7w52XA4+7+KlAjdiWJyJFYv349kydPZtCgQQogqfKiCaGNZvYf4FpgupkdFeVyIlLJ1q1bVxhAzZo1i3c5IqWKJkyuJdAJ6aXuvgM4Hvh1LIsSkbJbt24dL774Itdcc40CSBJGNEM57AXWAn2CvWLXd/dZMa9MRKL25ZdfFgZQ06ZN412OSNSiuTvuNmAcUD/4GGtmt8a6MBGJzpdffslLL72kAJKEFE0HptcDXYKjn2JmfwPmAv8Xy8JEpHRr165lypQpXHvttTRp0iTe5YiUWTTXhIwf75Aj+FzdEojEmQJIkkE0LaHngE/M7OXg6yuBZ2JWkYiUas2aNbz88ssMHjyYxo0bx7sckXIrNYTc/UEzmw10I9ACGu3un8W6MBGJTAEkyaSkXrS7AE8CpwJLgevdfUVlFVZe6r9Uktnq1at55ZVXGDJkCI0aNYp3OSJHrKRrQo8CtwP1gAeBf1VKRSISkQJIklFJIZTm7m+6+wF3nwycWFlFicihVq1apQCSpFTSNaFjzeyq4l67+5TYlSUiIatWreLVV19l6NChNGzYMN7liFSokkLoPaB/Ma8dUAiJxNgXX3zB1KlTFUCStEoa1G50ZRYiIocKBdCwYcNo0KBBvMsRiYlovieUUDS8tySDzz//nNdee00BJEkv6UJIJNF9/vnnvP766wwfPpxTTjkl3uWIxJRCSKQKWblyJdOmTWPYsGEKIEkJ0fSiXcvM/mhmTwVfn2Zml8e+NJHUogCSVBRNB6bPAQeAc4Ovc4B7Y1aRSApasWIF06ZN0yk4STnRhNCp7v53IBfA3fehXrRFKsyKFSuYPn06w4cP5+STT453OSKVKpprQgfNrCaB7wZhZqcSaBmJyBFavnw5M2bM4LrrruOkk06KdzkilS6aELoLmAk0MrNxwPnAqFgWJZIKFEAi0Q3l8KaZfQp0JXAa7jZ33xrzykSS2LJly5g5c6YCSFJeqSFkZt2DT3cFf7YyM9x9TuzKEkley5Yt44033mDEiBH85Cc/iXc5InEVzem4X4c9zwA6AwuBXjGpSCSJLV26lFmzZnHdddcpgESI7nRceCemmFkj4O8xq0gkSSmARA5Xnh4TcoA2FV1IRdG941IVLVmyhDfffJMRI0ZQv379eJcjUmVEc03o/wjenk3ge0UdgMUxrEkkqSiARIoXTUtoQdjzPGCCu38Yo3pEksrixYt56623GDlyJCeeqMGJRYoqMYTMLB3o7e7XVVI9Iklj8eLFvP322wogkRKU2G2Pu+cDJ5pZjUqqRyQpLFq0iLfffpsRI0YogERKUGxLyMwau/sGYD3woZlNBfaE3nf3B2Nfnkji+eyzz3j33XcZOXIkJ5xwQrzLEanSSjod9wpwNrAp+EgD6lZCTeXjpc8iEmsKIJGyKSmEDMDd/1RJtVQIDe8t8fLpp58ye/ZsBZBIGZQUQg3M7N/FvenuP49BPSIJ6dNPP+W9994jMzOTevXqxbsckYRRUgjtI9A9j4iUYOHChcyZM4eRI0cqgETKqKQQ2ubuWZVWiUgCUgCJHJmSQuhgpVUhkoAWLFjA+++/T2ZmJscff3y8yxFJSCV9T2hISQtaQMMKrueI6b4EqQzz589XAIlUgJJaQg+YWRrwKoFrQ1sIDOXQArgQuIjAqKs5sS5SpCqZP38+H374oQJIpAIUG0Lufo2ZtQKGAz8FTgb2AiuB6cBf3H1/pVQpUkXMmzePjz76iMzMTI477rh4lyOS8ErsO87dVwB/qKRaRKo0BZBIxSvPeEIiKeeTTz5h7ty5CiCRCqYQEinFJ598wscff8yoUaM49thj412OSFJRCImU4OOPP+aTTz4hMzNTASQSAyUO5QBgZi+Z2WXBO+WqPt2iLRVk7ty5CiCRGIsmWB4HhgGrzex+M2sZ45pE4m7u3LnMnz9fp+BEYqzUEHL3t9x9OIFhHdYDb5rZR2Y22syqx7pAkcr20UcfMX/+fDIzMznmmGPiXY5IUovqFJuZ1QNGATcAnwEPEwilN2NWmUgcfPjhhyxYsEABJFJJSr0xwcymAC2BMUB/d/8m+NYkM1sQy+JEKtOHH37IwoULGTVqFEcffXS8yxFJCdHcHfe0u08Pn2BmR7n7AXfvFKO6RCrVBx98wGeffaYAEqlk0ZyOuzfCtLnRrNzMLjWzL8xsjZn9roT5zjGzfDMbFM16RSpSKIAyMzMVQCKVrNiWkJmdBDQAaprZWfx48/PRQK3SVmxm6cCjQG8CnZzON7Opwa6Ais73N+CNcn0CkSPw/vvvs2jRIgWQSJyUdDquD4GbERoCD4ZN3wX8bxTr7gyscfcvAcxsInAFsKLIfLcCLwHnRFeySMWYM2cOS5YsYdSoUdStWzfe5YikpJJ60c4Csszsand/qRzrbgB8HfY6B+gSPoOZNQAGAr0oIYTM7EbgRoDGjRuXoxSRQ4UCKDMzUwEkEkclnY67zt3HAk3N7FdF33f3ByMsdsgqIkzzIq8fAn7r7vlWwmh07v4k8CRAp06diq5DpEzee+89li1bpgASqQJKOh1XO/izTjnXnQM0CnvdENhUZJ5OwMRgAJ0A9DOzPHd/pZzbFCnR7NmzWb58OZmZmdSpU94/bRGpKCWdjvtP8Olj7r6lHOueD5xmZs2AjQSGCx9WZBvNQs/N7HngdQWQxIoCSKTqieYW7Y/MbJaZXW9mUQ+k4u55wC0E7npbCbzg7svN7CYzu6mc9YqUmbvz7rvvsmLFCgWQSBVT6pdV3f00M+tMoCXzBzNbAUwMXi8qbdnpBIYCD5/2RDHzjoqqYpEycHdmz57NypUrGTlypAJIpIqJqu84d5/n7r8icNv190BWTKsSqQChFtDKlSvVAhKpoqIZT+hoM8s0sxnAR8A3BMJIpMoKBdAXX3xBZmYmtWvXLn0hEal00fQdtxh4BbjH3aPqrkckntydd955h1WrVjFy5EgFkEgVFk0INXf3hPluTknfN5Lk5+68/fbbrF69WgEkkgBK+rLqQ+7+C2CqmR0WQu4+IJaFiZRVKIDWrFlDZmYmtWqV2sWhiMRZSS2hMcGf/6iMQkSOhLvz1ltvsXbtWkaOHKkAEkkQJX1ZdWHwaQd3fzj8PTO7DXgvloWJRMvdefPNN1m3bp0CSCTBRHOLdmaEaaMquA6RcgkPoBEjRiiARBJMSdeEhhLoZqeZmU0Ne6susC3WhYmUxt2ZNWsW69evZ+TIkdSsWTPeJYlIGZV0TSj0naATgH+GTd8FLIllUSKlCQXQV199pQASSWAlXRP6CvgKOLfyyjlyukU7+bk7b7zxBhs2bGDEiBEKIJEEVtLpuA/cvZuZ7eLQcYAMcHfXWMhS6UIB9PXXXyuARJJASS2hbsGfGvVLqgR3Z+bMmeTk5DBixAgyMjLiXZKIHKFo+o471cyOCj7vaWY/N7NjY16ZSBh3Z8aMGWzcuFEBJJJEorlF+yUg38xaAM8AzYDxMa1KJEwogDZt2sR1112nABJJItGEUEFwgLqBwEPu/kvg5NiWVX66LSG5uDvTp09XAIkkqWg6MM0NfmcoE+gfnFY9diWJBIQCaPPmzYwYMYKjjjoq3iWJSAWLpiU0msBt2n9x93Vm1gwodVRVkSPh7kybNo3Nmzdz3XXXKYBEklQ0w3uvAH4e9nodcH8si5LU5u68/vrrbNmyRQEkkuRKDSEzOx+4G2gSnD/0PaHmsS1NUlF4AA0fPlwBJJLkorkm9AzwS2AhkB/bciSVuTuvvfYa27ZtUwCJpIhoQminu8+IeSWS0sIDaNiwYQogkRQRTQi9a2YPAFOAA6GJ7v5pzKoqF92cnajcnalTp/L9998zfPhwatSoEe+SRKSSRBNCXYI/O4VNc6BXxZdTAZRFCaWgoIDXXnuN7du3K4BEUlA0d8ddWBmFSOopKChg6tSp7Nixg2HDhimARFJQNH3H/cTMnjGzGcHXrczs+tiXJsksFEA7d+5UAImksGi+rPo88AZwSvD1KuAXMapHUkBBQQGvvvoqO3fuZOjQoQogkRQWTQid4O4vAAUAwX7kquyt2hrTrmoLBdCuXbvUAhKRqG5M2GNm9QgObGdmXYGdMa1KklJBQQGvvPIKu3fvZujQoVSvri4IRVJdNCH0K2AqcKqZfQicCAyKaVWSdEIBtGfPHgWQiBSK5u64T82sB3AGgRugv3D33JhXJkmjoKCAl19+mb179zJkyBAFkIgUKvaakJmdY2YnQeF1oI7AX4B/mtnxlVSfJLhQAO3bt08BJCKHKenGhP8ABwHMrDuBnrOzCVwPejL2pUmiKygoYMqUKezbt4/BgwcrgETkMCWdjkt39++DzwcDT7r7S8BLZrYo5pVJQsvPz2fKlCkcPHiQIUOGUK1aNJcfRSTVlNQSSjez0JHjIuCdsPd0RJFihQfQ4MGDFUAiUqySjg4TgPfMbCuwD3gfwMxaUIVv0TZ9USiu8vPzeemll8jLy1MAiUipij1CuPtfzOxt4GRglrt78K004NbKKE4SS3gAXXvttQogESlViUcJd/84wrRVsStHElV+fj4vvvgi+fn5CiARiVo03faIlCgUQAUFBQogESmTpDta6JpQ5crPz2fy5Mm4O9dcc40CSETKREcMKbdQAAFce+21pKenx7kiEUk0CiEpl7y8PCZPnkxaWhqDBg1SAIlIueiakJSZAkhEKopCSMokLy+PF154QQEkIhVCp+MkaqEAqlatGldffbUCSESOmFpCEpW8vDwmTZpE9erVFUAiUmEUQlKqUADVqFGDq666SgEkIhVGp+OkRKEAOuqooxg4cKACSEQqlEJIipWbm8ukSZPIyMjgqquuIi1NDWcRqVg6qkhEubm5TJw4kZo1ayqARCRmdGSRw4QCqFatWgwcOFABJCIxo9NxcohQANWuXZsrr7xSASQiMaUjjBTKzc1lwoQJCiARqTRqCQnwYwDVrVuXK664QgEkIpVCRxrh4MGDjB8/XgEkIpVOLaEUd/DgQSZMmMAxxxzDgAEDFEAiUql0xElhoRbQscceqwASkbjQUSdFhQLouOOOo3///gogEYkLnY5LQQcPHmTcuHEcf/zxDBgwQEOiJ4Hc3FxycnLYv39/vEuRFJWRkUHDhg2pXr16mZZLmhDSYTQ6Bw4cYPz48dSrV4/+/fsrgJJETk4OdevWpWnTpvqdSqVzd7Zt20ZOTg7NmjUr07I6B5NCDhw4wLhx4xRASWj//v3Uq1dPv1OJCzOjXr165WqJxzSEzOxSM/vCzNaY2e8ivD/czJYEHx+ZWftY1pPKQgF04oknKoCSlH6nEk/l/fuLWQiZWTrwKNAXaAUMNbNWRWZbB/Rw93bAn4EnY1VPKgsFUP369bn88st1sBKRKiOWLaHOwBp3/9LdDwITgSvCZ3D3j9x9e/Dlx0DDGNaTkg4cOMDYsWOpX78+l112mQJIksadd97JW2+9FdNtPP/882zatKnw9SOPPEKLFi0wM7Zu3VrisrfddhsNGjSgoKCgcNrdd9/NP/7xj0Pma9q0aeG6Nm/ezJAhQzj11FNp1aoV/fr1Y9WqVYete+bMmZxxxhm0aNGC+++/P+L2t2/fzsCBA2nXrh2dO3dm2bJlhe89/PDDtGnThtatW/PQQw8VTl+8eDHnnnsubdu2pX///vzwww8ArF+/npo1a9KhQwc6dOjATTfdVOJnL4tYhlAD4Ouw1znBacW5HpgR6Q0zu9HMFpjZgi1btlRgiclt//79jB07lpNOOkkBJEklPz+fe+65h4svvjim2ykaQueffz5vvfUWTZo0KXG5goICXn75ZRo1asScOXOi2pa7M3DgQHr27MnatWtZsWIFf/3rX/n2228PmS8/P5+bb76ZGTNmsGLFCiZMmMCKFSsOW99f//pXOnTowJIlS8jOzua2224DYNmyZTz11FPMmzePxYsX8/rrr7N69WoAbrjhBu6//36WLl3KwIEDeeCBBwrXd+qpp7Jo0SIWLVrEE088EdVnikYs746LdMTziDOaXUgghLpFet/dnyR4qq5Tp04R1yGHCgXQySefTL9+/RRAKWTHnXeTu2J5ha6zeqvWHHvP3SXOM3bsWP79739z8OBBunTpwmOPPcann37K9ddfz7x588jPz6dz585MmjSJrVu3cuedd1KvXj2++OILunfvzmOPPUZaWhqzZs3irrvu4sCBA5x66qk899xz1KlTh6ZNm/LTn/6UWbNmccsttzBz5kwuv/xyBg0aRNOmTRk2bBjvvvsuubm5PPnkk/z+979nzZo1/PrXvy78n/sDDzzACy+8wIEDBxg4cCB/+tOfWL9+PX379qVbt2589NFHNGjQgFdffZVp06axYMEChg8fTs2aNZk7dy5nnXVWVPvr3XffpU2bNgwePJgJEybQs2fPqJapXr36Ia2MDh06HDbfvHnzaNGiBc2bNwdgyJAhvPrqq7RqdejVjhUrVvD73/8egJYtW7J+/Xq+/fZbVq5cSdeuXalVqxYAPXr04OWXX+Y3v/lN4e8CoHfv3vTp04c///nPUX3m8oplSygHaBT2uiGwqehMZtYOeBq4wt23xbCelBEKoFNOOUUBJJVi5cqVTJo0iQ8//JBFixaRnp7OuHHjOOeccxgwYAB33HEHv/nNb7juuuto06YNEDiY/vOf/2Tp0qWsXbuWKVOmsHXrVu69917eeustPv30Uzp16sSDDz5YuJ2MjAw++OADhgwZclgNjRo1Yu7cuVxwwQWMGjWKF198kY8//pg777wTgFmzZrF69WrmzZvHokWLWLhwYWErZfXq1dx8880sX76cY489lpdeeolBgwbRqVMnxo0bx6JFi6hZs2bU+2PChAkMHTqUgQMH8vrrr5Obm1vqMsuWLaNjx46lzrdx40YaNfrx0NqwYUM2btx42Hzt27dnypQpQGBff/XVV+Tk5NCmTRvmzJnDtm3b2Lt3L9OnT+frrwMnrdq0acPUqVMBmDx5cuF0gHXr1nHWWWfRo0cP3n///VLrjFYsW0LzgdPMrBmwERgCDAufwcwaA1OAEe5++IlPKbP9+/czZswYGjRoQN++fRVAKai0FkssvP322yxcuJBzzjkHgH379lG/fn0gcO3mnHPOISMjg3//+9+Fy3Tu3Lnwf/NDhw7lgw8+ICMjgxUrVnD++ecDgS9Wn3vuuYXLDB48uNgaBgwYAEDbtm3ZvXs3devWpW7dumRkZLBjxw5mzZrFrFmzClszu3fvZvXq1TRu3JhmzZoVtjo6duzI+vXry70vDh48yPTp0/nXv/5F3bp16dKlC7NmzSrxlHhZ/p26H34yKNLyv/vd77jtttvo0KEDbdu25ayzzqJatWqceeaZ/Pa3v6V3797UqVOH9u3bU61aIAqeffZZfv7zn3PPPfcwYMAAatSoAcDJJ5/Mhg0bqFevHgsXLuTKK69k+fLlHH300VHXXZyYhZC755nZLcAbQDrwrLsvN7Obgu8/AdwJ1AMeC+7EPHfvFKuakl0ogBo2bMill16qAJJK4+5kZmZy3333Hfbe999/z+7du8nNzWX//v3Url0bOPzAaWa4O71792bChAkRtxNaNpKjjjoKgLS0tMLnodd5eXm4O7///e/52c9+dshy69evP2T+9PR09u3bV8onLt7MmTPZuXMnbdu2BWDv3r3UqlWLyy67jHr16vHNN98cMv+uXbs49thjad26NS+++GKp62/YsOEhLZScnBxOOeWUw+Y7+uijee6554DA76dZs2aFXyS9/vrruf766wH43//9Xxo2DNwT1rJlS2bNmgXAqlWrmDZtGhDYt6F91LFjR0499VRWrVpFp05HfriO6feE3H26u5/u7qe6+1+C054IBhDufoO7H+fuHYIPBVA57du3jzFjxtCoUSMFkFS6iy66iBdffJHvvvsOCATPV199BcCNN97In//8Z4YPH85vf/vbwmXmzZvHunXrKCgoYNKkSXTr1o2uXbvy4YcfsmbNGiBwAI90d1h59OnTh2effZbdu3cDgdNaoXqLU7duXXbt2lWm7UyYMIGnn36a9evXs379etatW8esWbPYu3cv3bt3Z+rUqYXrnDJlCu3btyc9PZ1evXpx4MABnnrqqcJ1zZ8/n/fee++Q9Z9zzjmsXr2adevWcfDgQSZOnFjYCgy3Y8cODh48CMDTTz9N9+7dC1suoc+9YcMGpkyZwtChQw+ZXlBQwL333lt4fWrLli3k5+cD8OWXX7J69erCVuyRSppue1JZKIAaN25Mnz59FEBS6Vq1asW9997LJZdcQkFBAdWrV+fRRx/lvffeo1q1agwbNoz8/HzOO+883nnnHdLS0jj33HP53e9+x9KlS+nevTsDBw4kLS2N559/nqFDh3LgwAEA7r33Xk4//fQjrvGSSy5h5cqVhaf36tSpw9ixY0lPTy92mVGjRnHTTTcV3pjw1FNP8fe//53NmzfTrl07+vXrx9NPP104/969e3njjTf4z3/+Uzitdu3adOvWjddee43Bgwdzyy230K1bN8yM+vXrFy5vZrz88sv84he/4P777ycjI4OmTZsecgs1QLVq1XjkkUfo06cP+fn5/PSnP6V169YAhXet3XTTTaxcuZKRI0eSnp5Oq1ateOaZZwrXcfXVV7Nt27bC39Nxxx0HBAL00UcfBeCqq65i9OjRAMyZM4c777yTatWqkZ6ezhNPPMHxxx9frt9DURbp/GJV1qlTJ1+wYMFh09f9sSlH7a3JKf9cGYeq4icUQE2aNOGSSy5RAKWolStXcuaZZ8a7jKjNnj2bf/zjH7z++uvxLkUqUKS/QzNbWNJZLvUdl8D27dtHdna2AkhEEpZOxyWovXv3MmbMGJo1a0bv3r0VQJJQevbsGdV3ZyT5KYQSUCiAmjdvzsUXX6wAEpGEpdNxCWbv3r1kZ2crgEQkKagllEBCAdSiRQsuuugiBZCIJDyFUILYs2cP2dnZnH766fTq1UsBJCJJQafjEoACSORw8RjKYfjw4Zxxxhm0adOGn/70pyX2CZeIQzksWrSIrl270qFDBzp16sS8efMAyM3NJTMzk7Zt23LmmWdG7Bmj3Nw9oR4dO3b0SL68o4lv/FXLiO8lst27d/tjjz3mb7/9thcUFMS7HKmiVqxYEe8SKlVeXl6lbKdHjx4+f/78wtfTpk3zgoICLygo8CFDhvhjjz0Wcbn8/Hxv1KiRd+nSxd99993C6XfddZc/8MADh8zbpEkT37JlixcUFHjXrl398ccfL3zvs88+8zlz5hwyf15enjdv3tzXrl3rBw4c8Hbt2vny5csPq+H222/3u+++293dV65c6b169XJ396VLl3rr1q19z549npub6xdddJGvWrXK3d179+7t06dPL/ysPXr0cHf3cePG+eDBg93dfc+ePd6kSRNft27dYduM9HcILPASjuk6HVeF7dmzh6ysLM4880x69uypFpBE5V8zVrJqc9m6minN6SfV5Zd9S/4ybCoM5dCvX7/Cz9u5c2dycnIi7otEHcrBzAoHstu5c2dhn3Rmxp49e8jLy2Pfvn3UqFGjQjovBZ2Oq7J2796tAJKEkWpDOeTm5jJmzBguvfTSiPsjUYdyeOihh/j1r39No0aNuP322wtPuw0aNIjatWtz8skn07hxY26//fYK67ZHLaEqaPfu3WRnZ9OqVSt9oU/KrLQWSyyk2lAO//M//0P37t254IILDnsvkYdyePzxx/nXv/7F1VdfzQsvvMD111/PW2+9xbx580hPT2fTpk1s376dCy64gIsvvrhCOjFVCFUxoRZQ69atFUCSMDyFhnL405/+xJYtWw7ppDRcIg/lkJWVxcMPPwzANddcww033ADA+PHjufTSS6levTr169fn/PPPZ8GCBRUSQjodV4Xs2rWLrKws2rRpowCShJIqQzk8/fTTvPHGG0yYMIG0tMiHz0QeyuGUU04p3N4777zDaaedBkDjxo155513cHf27NnDxx9/TMuWLUvcd9FKqpaQJ/Blk1AAtW3blh49esS7HJEySZWhHG666SaaNGlSuI6rrrqq8JoTJP5QDk899RS33XYbeXl5ZGRk8OSTTwJw8803M3r0aNq0aYO7M3r0aNq1a1eu30NRSTOUw5d/bMpR+2rS4B+JN5RDKIDatWtH9+7d412OJCAN5SBVQXmGckiqllAiCgVQ+/btI17kFBFJZgqhOPrhhx/IysqiQ4cOCiBJKRrKQUIUQnESCqCzzjqLbt26xbscEZG40N1xcaAAEhEJUEuoku3cuZOsrCw6duxY+IU8EZFUpRCqRKEA6tSpE+edd168yxERiTudjqskCiCRihWPoRyuv/562rdvT7t27Rg0aFDhF18jScShHBYvXsy5555L27Zt6d+/f2FnpuPGjaNDhw6Fj7S0NBYtWlT8jiuLkrrYroqP4oZyWHtHE8/5f1VzKIft27f7ww8/7B999FG8S5EkpaEcYqPoUA47d+4sfP7LX/7S77vvvojLJepQDp06dfLZs2e7u/szzzzjd9xxx2HrXbJkiTdr1izi59ZQDlXQjh07yMrKonPnzod0xCgSK08teZJ1O7+s0HU2O6Y5/9XuxhLnSYWhHELd3rg7+/btK7bj0UQdyiH0uwDo3bs3ffr04c9//vMh6w31EF5RdDouhkIB1KVLFwWQJLVUGsph9OjRnHTSSXz++efceuutEfdHog7l0KZNG6ZOnQrA5MmTD+koNWTSpEkVGkJJ0xKqat3GhQKoa9eudOnSJd7lSAoprcUSC6k0lMNzzz1Hfn4+t956K5MmTWL06NGHvJ/IQzk8++yz/PznP+eee+5hwIAB1KhR45B1fvLJJ9SqVavwPxIVIWlCqCrZvn07WVlZnHvuuQogSQmeQkM5hOYZPHgwDzzwwGEhlMhDObRs2ZJZs2YBsGrVKqZNm3bIOidOnFihrSDQ6bgKFwqg8847TwEkKSMVhnJw98K63J3XXnst4nAGiTyUQ2h6QUEB99577yHXpwoKCpg8eXLEU6FHQi2hChQeQJ07d453OSKVJhWGcvjwww/JzMzkhx9+wN1p3749jz/++CHzJ/pQDhMmTODRRx8FAsNUhLfy5syZQ8OGDStkILtwSTOUw7o/NqVGHIdy+P7778nOzub8888vPC8uUlk0lINUBeUZykGn4yrA999/T1ZWFt26dVMAiYiUgU7HHaFQAF1wwQV06lRs2ItIGA3lICEKoSOwbds2srOz6d69e1T394uIyKF0Oq6cFEAiIkdOIVQO27ZtIysrix49eiiARESOgE7HldHWrVvJzs6mZ8+enH322fEuR0QkoaklVAahALrwwgsVQCJxFo+hHEJuvfVW6tSpU+KyyTSUw7x58wqHcWjfvj0vv/xyiZ+9LBRCUQoFUK9evQr7nhKR+MjPz+eee+7h4osvjul2IoXQggUL2LFjR4nLFRQU8PLLL9OoUaPCTlJL4+4MHDiQnj17snbtWlasWMFf//pXvv3220Pmy8/P5+abb2bGjBmsWLGCCRMmsGLFisPW99e//pUOHTqwZMkSsrOzue2224BAR6lPPfUU8+bNY/Hixbz++uusXr0agBtuuIH777+fpUuXMnDgQB544AEg0LHpggULWLRoETNnzuRnP/sZeXl5UX2u0uh0XBS2bNnCmDFj6NWrV8Su1UWqkg+fXsC2dd9X6DrrNTue828o+SsIqTCUQ40aNfj1r3/N+PHjS2wNJNtQDqF5Afbv31+mDldLo5ZQKUIBdNFFFymARIqRKkM5PPLIIwwYMICTTz65xP2RjEM5fPLJJ7Ru3Zq2bdvyxBNPFPa8faTUEirBd999x5gxY7j44otp3759vMsRiUppLZZYSIWhHDZt2sTkyZOZPXt2ifsiWYdy6NKlC8uXL2flypVkZmbSt29fMjIyoq67OAqhYoQCqHfv3rRr1y7e5YhUaakwlMNnn33GmjVraNGiBRDorLRFixaFPWuHJPNQDgBnnnkmtWvXZtmyZRXSS4xOx0WgABIpm1QYyuGyyy5j8+bNhUM01KpV67AAguQcymHdunWFNyJ89dVXfPHFFzRt2rTEfRcttYSK+Pbbbxk7diyXXHJJ4f9kRKRkqTCUw9y5cwuH+C5Osg7l8MEHH3D//fdTvXp10tLSeOyxxzjhhBNK3efR0FAOYRRAkqg0lINUBRrK4Qh8++23jBkzhj59+iiAREQqiU7HEfiW8tixY7n00ksLbx8VkdjRUA4SkvIhFAqgvn37Fp5XFRGRypHSIfTNN98wbtw4BZCISJykbAiFAqhfv36HdXchIiKVIyVvTFAAiYhUDSkXQps2bWLcuHFcdtllCiCRBBaPoRxGjRpV2MVPhw4dWLRoUbHLJtNQDuPGjSv8zB06dCAtLa3Ez14m7p5Qj44dO3okX97RxHP+X8uI74Vs3LjRH3jgAV+xYkWJ84kkmlT7m87Ly6uU7fTo0cPnz59f+DozM9MnT55c6nL5+fneqFEj79Kli7/77ruF0++66y5/4IEHDpm3SZMmvmXLFi8oKPCuXbv6448/XvjeZ5995nPmzDlk/ry8PG/evLmvXbvWDxw44O3atfPly5cfVsPtt9/ud999t7u7r1y50nv16uXu7kuXLvXWrVv7nj17PDc31y+66CJftWqVu7t36tTJZ8+e7e7uzzzzjN9xxx2HrXfJkiXerFmziJ870t8hsMBLOKanzDWhTZs2MX78eC6//HJatmwZ73JEYmfGL2Dzoopd50kdoO9DJc6SCkM5RCvZhnIIF+ohvKKkxOm4jRs3Mn78ePr3768AEomBVBnKAeAPf/gD7dq145e//GVh10JFJeNQDiGTJk2q0BBK+pZQKIAGDBjAGWecEe9yRGKvlBZLLKTCUA4A9913HyeddBIHDx7kxhtv5G9/+1thyIUk61AOEBhTqFatWhX6pf6kDqGcnBwmTJigABKJMU+BoRyAwsHsjjrqKEaPHn3YTQaQ3EM5TJw4sUJbQZDEp+NCAXTFFVcogERiLBWGcgAKA8TdeeWVVyK2CJJxKIfQtMmTJ0c8FXokkrIl9PXXXzNx4kSuvPJKTjvttHiXI5L0UmUoh+HDh7NlyxbcnQ4dOhQOnRCSrEM5AMyZM4eGDRsWnkKtKDEdysHMLgUeBtKBp939/iLvW/D9fsBeYJS7f1rSOksbyqHgtlkKIEk5GspBqoIqNZSDmaUDjwJ9gVbAUDMr+u3QvsBpwceNwONHut2JEycycOBABZCISAKI5TWhzsAad//S3Q8CE4ErisxzBZAd/E7Tx8CxZnZyubYWbNANHDiwcAx4EamaevbsqVaQALENoQZA+E3mOcFpZZ0HM7vRzBaY2YItW7ZE3Nh31U7k62onKIAkZcXy1LpIacr79xfLGxMi3fhetMpo5sHdnwSehMA1oUgb63L3/LLWJ5I0MjIy2LZtG/Xq1SvTd05EKoK7s23bNjIyMsq8bCxDKAdoFPa6IbCpHPOISCkaNmxITk4OxZ0pEIm1jIyMwu8blUUsQ2g+cJqZNQM2AkOAYUXmmQrcYmYTgS7ATnf/BhEpk+rVqxd+EVEkkcQshNw9z8xuAd4gcIv2s+6+3MxuCr7/BDCdwO3Zawjcoj26uPWJiEjyiemXVd19OoGgCZ/2RNhzB26OZQ0iIlJ1JW23PSIiUvXFtMeEWDCzLcBXxbx9ArC1EstJFNovxdO+iUz7JTLtl8hK2i9N3P3E4hZMuBAqiZktKKl7iFSl/VI87ZvItF8i036J7Ej2i07HiYhI3CiEREQkbpIthJ6MdwFVlPZL8bRvItN+iUz7JbJy75ekuiYkIiKJJdlaQiIikkAUQiIiEjcJGUJmdqmZfWFma8zsdxHeNzP7d/D9JWZ2djzqrGxR7Jfhwf2xxMw+MrP28aizspW2X8LmO8fM8s1sUGXWF0/R7Bsz62lmi8xsuZm9V9k1xkMU/5aOMbPXzGxxcL+kRJdjZvasmX1nZsuKeb/sx153T6gHgX7o1gLNgRrAYqBVkXn6ATMIDBXRFfgk3nVXkf1yHnBc8Hlf7ZfD5nuHQDdTg+Jdd1XZN8CxwAqgcfB1/XjXXUX2y/8Cfws+PxH4HqgR79orYd90B84GlhXzfpmPvYnYEqrcEVsTR6n7xd0/cvftwZcfExg6I9lF8/cCcCvwEvBdZRYXZ9Hsm2HAFHffAODuqbB/otkvDtS1wOBNdQiEUF7llln53H0Ogc9anDIfexMxhCpsxNYkU9bPfD2B/7Eku1L3i5k1AAYCT5BaovmbOR04zsxmm9lCMxtZadXFTzT75RHgTALjny0FbnP3gsopr0or87E3pr1ox0iFjdiaZKL+zGZ2IYEQ6hbTiqqGaPbLQ8Bv3T0/xUYljWbfVAM6AhcBNYG5Zvaxu6+KdXFxFM1+6QMsAnoBpwJvmtn77v5DjGur6sp87E3EENKIrZFF9ZnNrB3wNNDX3bdVUm3xFM1+6QRMDAbQCUA/M8tz91cqpcL4ifbf0lZ33wPsMbM5QHsgmUMomv0yGrjfAxdC1pjZOqAlMK9ySqyyynzsTcTTcYUjtppZDQIjtk4tMs9UYGTwTo2upMaIraXuFzNrDEwBRiT5/2TDlbpf3L2Zuzd196bAi8D/pEAAQXT/ll4FLjCzamZWi8AIyCsruc7KFs1+2UCgdYiZ/QQ4A/iyUqusmsp87E24lpBrxNaIotwvdwL1gMeC/+vP8yTvETjK/ZKSotk37r7SzGYCS4AC4Gl3j3h7brKI8m/mz8DzZraUwCmo37p70g/xYGYTgJ7ACWaWA9wFVIfyH3vVbY+IiMRNIp6OExGRJKEQEhGRuFEIiYhI3CiEREQkbhRCIiISNwohOWKl9awbNt8fgj0OLwn2ytylguuYbmbHBp//3MxWmtk4MxtQUu/Zwfk/Cv5sambDyrHts8zs6eDzUWa2JfgZF5lZdgnLjTKzR8q6vQjryQ9ua5mZTQ5+p6csy59iZi8Gn3cws35h75W6/6LcRvh++dzMfhnlMqdEMd8/zKzXkdYolU+3aMsRM7PuwG4CHRe2KWaec4EHgZ7ufsDMTiDQ63BMerIws88J9AqxrozL9QRud/fLy7jcZOBed19sZqOATu5+SxTLRT1vKevZ7e51gs/HAQvd/cFyrqtCaippvWZWD/gCOMvdvy5hmdkEfh8LSll3E+Apd7+kAkuWSqCWkByxKHrWBTiZQPcvB4LLbA0FkJmtN7O/mdm84KNFcPqJZvaSmc0PPs4PTq9jZs+Z2dJgq+rqsPWcYGZPEOiGf6qZ/TK8tWFmPzGzly0wDsxiMzsvOH13sM77CfQQsCi47Ptm1iH0IczsQwt0fUTYtLpAO3dfXNyHN7P+ZvaJmX1mZm8Fv2VfdJ5rgi2ZxRboHgczSzezB4Kff4mZ/ayU/QzwPtDCzI43s1eCy30cqtvMeoS10j4zs7rBFuAyC/QQcA8wOPj+4ND+s8AYOuvNLC24nlpm9rWZVTezU81spgU6OX3fzFqWVGCwy6g1BP4uMLM7g59xmZk9aQGDCHSpNC5YS00z62hm7wW384YFe2h296+AemZ2UhT7R6qSeIxJoUfyPYCmFDPGSPD9OgQ6fFwFPAb0CHtvPfCH4PORwOvB5+OBbsHnjYGVwed/Ax4KW/64sPWcEOH5KOCR4PNJwC+Cz9OBY4LPdwd/9gxtP/g6M7QtAj1KL4jw2S4EXgp7PQrYEvy8iwh8a/w4fjzzcAPwzwi1LQUaBJ8fG/x5I3BH8PlRwAKgWYQaQvVXI9DVzn8D/wfcFZzeC1gUfP4acH7Y76Va+O8vvKYINb4KXBh8PphADwoAbwOnBZ93Ad6JUGP4ehoH901G8PXxYfONAfoHn88m0HqCwDfzPwJODNv+s2HLPQVcHe9/C3qU7ZFw3fZIYnL33WbWEbiAwEF7kpn9zt2fD84yIeznv4LPLwZa2Y89Wx8dbHVcTKA/r9C6Q2MkRaMXgaDD3fOBnaXMPxn4o5n9Gvgp8HyEeU4mEDrhJnnY6Swza0vgM59MYKC0SKcJPyTQFcwLBPr4A7gEaGc/jvZ6DHBahOVrmtmi4PP3gWeAT4CrAdz9HTOrZ2bHBLfzYPC03RR3z7Hoew+fRODg/y6B38FjZlaHwICJk8PWc1Qxyw+2QC/uZwD/5e77g9MvNLPfALWA44HlBMIy3BlAGwI9VkPgPxHh/ZJ9B5R6/UiqFoWQxISZNeLHg8gTHuiHLJ/A/2xnW6DPrUx+PKiHX5wMPU8DznX3fUXWbVTS0BzuvtfM3iQwWNe1BE4PFbUPyChlVf8HPOjuUy1w3enuCNu6yQI3a1wGLAqeBjTgVnd/o5T173P3DuETLHKyuLvfb2bTCPTx9bGZXQzsjzBvJFOB+8zseAJDPLwD1AZ2FN1+MSZ54JrQucA0M5sB7CDQOu7k7l+b2d1E3p8GLHf3c4tZdwaB34UkEF0Tkphw96/dvUPw8YSZnWFmp4XN0gH4Kuz14LCfc4PPZwHhrYkOxUw/rgylvU3gVFXoesvRRd7fBdQtMu1p4N/AfHePdO1rJdCilO0eA2wMPs+MNIOZnerun7j7ncBWAl3ivwH8t5lVD85zupnVLmVbIXOA4cHlehK4JvdDcDtL3f1vBE7vFb1+E2kfAIEWLYHhCh4mcNoy3wNj6Kwzs2uC2zIza19SYe4+l8Bpt9v4MXC2BltVg8JmDa/lC+DEYIARvBbVOmze04Gk7lw1GSmE5IhZoGfducAZZpZjZtdHmK0OkGVmK8xsCdCKQ1sDR5nZJwQOSqFbd38OdApeWF8B3BScfi+B0T6XmdliAqf3onUbgVM/S4GFQOsi7y8B8oI3B/wSwN0XAj8Az0Vaobt/DhwTPFVYnLsJnK56n0DARPKABW62WEYgQBYTCMAVwKfB6f8h+jMYdxPcfwRuuAiF3y/C9t0+Dh9h910Cp0EXmdlgDjcJuC74M2Q4cH1wncuJPIR6UX8jcL0sn8D1nKXAKwSGUgh5HngieKoxnUBA/S24nUUETgMSDOkWBEJVEohu0Za4M7P1BE7FVMmu8C3wPZXZQEsvZgjnYGDtcvenK7M2CTCzgcDZ7v7HeNciZaOWkEgJzGwkgQv8fygugIIeBw5UTlUSQTXgn/EuQspOLSEREYkbtYRERCRuFEIiIhI3CiEREYkbhZCIiMSNQkhEROLm/wOLTbv/KBLEyQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 424.8x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import torch\n",
    "\n",
    "palette = plt.cm.get_cmap('Set1')\n",
    "\n",
    "fig, ax = plt.subplots(1, 1, figsize=(5.9, 6), tight_layout=True)\n",
    "\n",
    "pltroot = 'roc_curve/cta3d'\n",
    "ax.plot([0,1],[0,1], color='gray', linewidth=1)\n",
    "for i, (file,auc) in enumerate(zip(sorted(os.listdir(pltroot)), [0.9995,0.9988,0.9997,0.9993,0.9997])):\n",
    "    x,y = torch.load(os.path.join(pltroot, file))\n",
    "    ax.plot(x, y, label=f'experiment{i+1} AUC {auc:.4f}', color=palette(i))\n",
    "\n",
    "ax.set_xlabel('1-Specificity (False Positive Rate)')\n",
    "ax.set_ylabel('Sensitivity (True Positive Rate)')\n",
    "ax.set_xlim(-0.02, 1.02)\n",
    "ax.set_ylim(-0.02, 1.02)\n",
    "#plt.legend(bbox_to_anchor=(1.04, 1), loc=\"upper left\")\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.title('3D Net for CTA')\n",
    "fig.savefig('roc-cta3d.png', dpi=500)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cb62efc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "26c901ce2aee1eb793855eaea4772ff853229105a532a42878eab53695b6b942"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
